1 <?php
2 $currDir = dirname(__FILE__);
3 require("{$currDir}/incCommon.php");
4
5 $GLOBALS['DEBUG_MODE'] = false;
6 $csv = new CSV($_REQUEST);
7
8 class CSV{
9 private $curr_dir,
10 $curr_page,
11 $lang, /* translation text */
12 $request, /* assoc array that stores $_REQUEST */
13 $error_back_link,
14 $max_batch_size, /* max # of non-empty lines to insert per batch */
15 $max_data_length, /* max length of csv data in read per batch in bytes */
16 $initial_ts; /* initial timestamp */
17
18 public function __construct($request = array()){
19 global $Translation;
20
21 $this->curr_dir = dirname(__FILE__);
22 $this->curr_page = basename(__FILE__);
23 $this->max_batch_size = 500;
24 $this->max_data_length = 0.5 * 1024 * 1024;
25 $this->initial_ts = microtime(true);
26 $this->lang = $Translation;
27
28 /* back link to use in errors */
29 $this->error_back_link = '' .
30 '<div class="text-center vspacer-lg"><a href="' . $this->curr_page . '" class="btn btn-danger btn-lg">' .
31 '<i class="glyphicon glyphicon-chevron-left"></i> ' .
32 $this->lang['back and retry'] .
33 '</a></div>';
34
35 /* process request to retrieve $this->request, and then execute the requested action */
36 $this->process_request($request);
37 call_user_func_array(array($this, $this->request['action']), array());
38 }
39
40 protected function debug($msg, $html = true){
41 if($GLOBALS['DEBUG_MODE'] && $html) return "<pre>DEBUG: {$msg}</pre>";
42 if($GLOBALS['DEBUG_MODE']) return " [DEBUG: {$msg}] ";
43 return '';
44 }
45
46 protected function elapsed(){
47 return number_format(microtime(true) - $this->initial_ts, 3);
48 }
49
50 protected function process_request($request){
51 /* action must be a valid controller, else set to default (show_load_form) */
52 $controller = isset($request['action']) ? $request['action'] : false;
53 if(!in_array($controller, $this->controllers())) $request['action'] = 'show_load_form';
54
55 $this->request = $request;
56 }
57
58 /**
59 * discover the public functions in this class that can act as controllers
60 *
61 * @return array of public function names
62 */
63 protected function controllers(){
64 $csv = new ReflectionClass($this);
65 $methods = $csv->getMethods(ReflectionMethod::IS_PUBLIC);
66
67 $controllers = array();
68 foreach($methods as $mthd){
69 $controllers[] = $mthd->name;
70 }
71
72 return $controllers;
73 }
74
75 /**
76 * function to show form for uploading a CSV file or choosing one from the 'csv' folder
77 */
78 public function show_load_form(){
79 /* get list of available CSV files */
80 $csv_files = $this->csv_files("{$this->curr_dir}/csv");
81
82 /* prepare tables drop-down */
83 $tables = getTableList();
84 $tables_dropdown = htmlSelect('table', array_keys($tables), array_values($tables), '');
85 $tables_dropdown = str_replace('<select ', '<select class="form-control input-lg" ', $tables_dropdown);
86 $tables_dropdown = preg_replace('/(<select .*?>)/i', "\$1<option value=\"\">{$this->lang['select a table']}</option>", $tables_dropdown);
87
88 echo $this->header();
89 ?>
90 <form method="post" action="<?php echo $this->curr_page; ?>" enctype="multipart/form-data">
91 <input type="hidden" name="action" value="upload">
92 <?php echo csrf_token(); ?>
93 <div class="page-header"><h1><?php echo $this->lang['import CSV to database']; ?></h1></div>
94
95 <h4><?php echo $this->lang['import CSV to database page']; ?></h4>
96
97 <div class="panel panel-success">
98 <div class="panel-heading">
99 <h3 class="panel-title"><i class="glyphicon glyphicon-th hspacer-md"></i> <?php echo "<b>{$this->lang['step 1']}</b> {$this->lang['table']}"; ?></h3>
100 </div>
101 <div class="panel-body">
102 <div class="form-group">
103 <?php echo $tables_dropdown; ?>
104 <span class="help-block"><?php echo $this->lang['populate table from CSV']; ?></span>
105 </div>
106 </div>
107 </div>
108
109 <div class="panel panel-success">
110 <div class="panel-heading">
111 <h3 class="panel-title"><i class="glyphicon glyphicon-upload hspacer-md"></i><?php echo "<b>{$this->lang['step 2']}</b> {$this->lang['upload or choose csv file']}"; ?></h3>
112 </div>
113 <div class="panel-body">
114 <label for="upload_csv" class="btn btn-primary btn-lg">
115 <i class="glyphicon glyphicon-upload"></i> <?php echo $this->lang['choose csv upload']; ?>
116 </label>
117
118 <span class="hspacer-lg"></span>
119 <span id="csv_file_name"><?php echo $this->lang['no file chosen yet']; ?></span>
120 <input type="file" name="upload_csv" id="upload_csv" accept=".csv, text/csv">
121 <button type="submit" class="btn btn-success btn-lg hspacer-lg hidden" id="start_upload"><i class="glyphicon glyphicon-upload"></i> <?php echo $this->lang['start upload']; ?></button>
122
123 <?php if(count($csv_files)){ ?>
124 <hr>
125 <div class="panel panel-primary">
126 <div class="panel-heading">
127 <h3 class="panel-title"><i class="glyphicon glyphicon-folder-open hspacer-md"></i> Open an existing CSV file</h3>
128 </div>
129 <div class="panel-body hidden">
130 <div class="row" id="existing-csv-files">
131 <?php foreach($csv_files as $csv_file){ ?>
132 <div class="col-lg-2 col-md-3 col-sm-4 col-xs-6 csv-file">
133 <button type="button" class="btn btn-link invisible delete-csv" data-csv="<?php echo html_attr($csv_file); ?>" title="<?php echo html_attr($this->lang['delete']); ?>"><i class="glyphicon glyphicon-trash text-danger"></i></button>
134 <a href="<?php echo $this->curr_page; ?>?csv=<?php echo urlencode($csv_file); ?>&action=show_preview&table=">
135 <i class="glyphicon glyphicon-file text-success"></i> <?php echo $csv_file; ?>
136 </a>
137 </div>
138 <?php } ?>
139 </div>
140 </div>
141 </div>
142 <?php } ?>
143 </div>
144 </div>
145 </form>
146
147 <script>
148 $j(function(){
149 /* function to highlight table drop-down as required */
150 var highlight_dropdown = function(){
151 $j('#table').focus().parent().addClass('has-error');
152 return false;
153 };
154
155 /* function to unhighlight table drop-down */
156 var unhighlight_dropdown = function(){
157 $j('#table').parent().removeClass('has-error');
158 };
159
160 /* function to update csv file links */
161 var update_csv_links = function(){
162 var table = $j('#table').val();
163 var csrf_token = $j('#csrf_token').val();
164
165 if(table.length) unhighlight_dropdown();
166
167 $j('.csv-file a').each(function(){
168 var href = $j(this).attr('href');
169 href = href.replace(/(action=show_preview).*$/, '$1');
170 href += '&csrf_token=' + csrf_token;
171 href += '&table=' + table;
172 $j(this).attr('href', href);
173 });
174 }
175
176 /* validate and display name of selected-for-upload CSV file */
177 $j('#upload_csv').change(function(){
178 var csv_file = $j(this).val();
179 if(!csv_file.match(/\.csv$/i)){
180 $j('#csv_file_name').html(
181 '<span class="text-danger bg-danger">' +
182 '<i class="glyphicon glyphicon-remove"></i> ' +
183 '<?php echo $this->lang['invalid csv file selected']; ?>' +
184 '</span>');
185 $j(this).val('');
186 $j('#start_upload').addClass('hidden');
187 return false;
188 }
189 $j('#csv_file_name').html(
190 '<span class="bg-success text-success">' +
191 '<i class="glyphicon glyphicon-ok"></i> ' + csv_file +
192 '</span>'
193 );
194 $j('#start_upload').removeClass('hidden');
195 });
196
197 /* toggle panel-body and panel-footer on clicking panel-title */
198 $j('.panel-heading').click(function(){
199 var panel = $j(this).parent();
200 panel.find('.panel-body').toggleClass('hidden');
201 panel.find('.panel-footer').toggleClass('hidden');
202 });
203
204 /* hover effect for existing csv files */
205 var highlighter = 'success';
206 $j('.row').on('mousemove', '.csv-file', function(){
207 $j('.csv-file').removeClass('text-' + highlighter + ' bg-' + highlighter);
208 $j('.delete-csv').addClass('invisible');
209 $j(this).addClass('text-' + highlighter + ' bg-' + highlighter);
210 $j(this).find('button').removeClass('invisible');
211 }).mouseout(function(){
212 $j('.csv-file').removeClass('text-' + highlighter + ' bg-' + highlighter);
213 $j('.delete-csv').addClass('invisible');
214 });
215
216 /* on changing table, update csv file links */
217 $j('#table').change(function(){
218 update_csv_links();
219 });
220
221 /* on submitting csv, make sure a table is selected */
222 $j('#start_upload').click(function(){
223 var table = $j('#table').val();
224 if(!table.length){
225 return highlight_dropdown();
226 }
227 });
228 $j('#existing-csv-files').on('click', '.csv-file a', function(){
229 var table = $j('#table').val();
230 if(!table.length){
231 return highlight_dropdown();
232 }
233 });
234
235 $j('#existing-csv-files').on('click', '.delete-csv', function(){
236 var del_btn = $j(this);
237 var csv = del_btn.data('csv');
238 var msg = '<?php echo html_attr($this->lang['sure delete csv']); ?>';
239 msg = msg.replace(/\[CSVFILE\]/, '"' + csv + '"');
240
241 var fail_delete = function(elm){
242 elm.popover({
243 placement: 'auto bottom',
244 title: '<?php echo html_attr($this->lang['errors occurred']); ?>',
245 content: '<span class="text-danger"><?php echo html_attr($this->lang['couldnt delete csv file']); ?></span>',
246 trigger: 'manual',
247 container: 'body',
248 html: true
249 }).popover('show').on('shown.bs.popover', function(){
250 setTimeout(function(){
251 elm.popover('hide').popover('destroy');
252 }, 3000);
253 });
254 };
255
256 if(confirm(msg)){
257 var url = '<?php echo $this->curr_page; ?>?action=delete_csv&csv=' + encodeURIComponent(csv);
258 $j.ajax(url)
259 .done(function(data){
260 if(data.deleted){
261 del_btn.parent().remove();
262 }else{
263 fail_delete(del_btn.parent());
264 }
265 })
266 .fail(function(){
267 fail_delete(del_btn.parent());
268 });
269 }
270 });
271 })
272 </script>
273
274 <style>
275 .csv-file{
276 overflow: hidden;
277 white-space: nowrap;
278 font-size: 1.2em;
279 padding-top: 0.4em;
280 }
281 label[for=upload_csv]{
282 cursor: pointer;
283 }
284 #upload_csv{
285 opacity: 0;
286 position: absolute;
287 z-index: -1;
288 }
289 .panel-heading{
290 cursor: pointer;
291 }
292 .panel-title{
293 font-size: 1.4em;
294 }
295 </style>
296 <?php
297 echo $this->footer();
298 }
299
300 public function delete_csv(){
301 $deleted = false;
302 @header('Content-type: application/json');
303
304 $csv_folder = "{$this->curr_dir}/csv/";
305 $csv = $this->get_csv();
306 if($csv && @unlink($csv_folder . $csv)) $deleted = true;
307
308 echo json_encode(array('deleted' => $deleted));
309 }
310
311 private function csv_files($dir){
312 $csv_files = array();
313
314 if(!is_dir($dir)) @mkdir($dir);
315
316 $d = dir($dir);
317 while(false !== ($entry = $d->read())){
318 if(preg_match('/\.csv$/i', $entry)) $csv_files[] = urldecode($entry);
319 }
320 $d->close();
321 return $csv_files;
322 }
323
324 /**
325 * function to handle csv file upload request by validating and saving into the csv folder
326 */
327 public function upload(){
328 if(!csrf_token(true)){
329 echo $this->header();
330 echo errorMsg("{$this->lang['csrf token expired or invalid']}<br>{$csv_file}{$this->error_back_link}" . $this->debug(__LINE__));
331 echo $this->footer();
332
333 return;
334 }
335
336 $csv_file = getUploadedFile('upload_csv', PHP_INT_MAX, 'csv', true);
337 $table = $this->get_table();
338 if(!$table) return;
339
340 if(!$csv_file || !is_readable($csv_file)){
341 echo $this->header();
342 echo errorMsg("{$this->lang['csv file upload error']}<br>{$csv_file}{$this->error_back_link}" . $this->debug(__LINE__));
343 echo $this->footer();
344
345 return;
346 }
347
348 echo $this->header();
349 ?>
350 <div class="alert alert-success vspacer-lg"><h2>
351 <i class="glyphicon glyphicon-ok"></i>
352 <?php echo $this->lang['please wait and do not close']; ?>
353 </h2></div>
354 <script>
355 $j(function(){
356 window.location = '<?php echo $this->curr_page; ?>?action=show_preview&csv=<?php echo urlencode(basename($csv_file)); ?>&table=<?php echo urlencode($table); ?>';
357 });
358 </script>
359 <?php
360 echo $this->footer();
361 }
362
363 /**
364 * @brief retrieve and validate the csv file specified in the request parameter 'csv'
365 *
366 * @param [in] $options optional assoc array of options ('htmlpage' => bool, displaying errors as html page)
367 * @return csv filename if valid, false otherwise.
368 */
369 protected function get_csv($options = array()){
370 $csv_ok = true;
371
372 $csv = $this->request['csv'];
373 if(!$csv) $csv_ok = false;
374
375 if($csv_ok){
376 $csv = basename($csv);
377 if(!is_readable("{$this->curr_dir}/csv/{$csv}")) $csv_ok = false;
378 }
379
380 if(!$csv_ok){
381 if(isset($options['htmlpage'])){
382 echo $this->header();
383 echo errorMsg($this->lang['csv file upload error'] . $this->error_back_link . $this->debug(__LINE__));
384 echo $this->footer();
385 }
386 return false;
387 }
388
389 return $csv;
390 }
391
392 /**
393 * @brief Retrieve and validate name of table used for importing data
394 *
395 * @param [in] $silent (optional) boolean indicating no output to client if true, useful in ajax requests for example.
396 * @return table name, or false on error.
397 */
398 protected function get_table($silent = false){
399 $table_ok = true;
400
401 $table = $this->request['table'];
402 if(!$table) $table_ok = false;
403
404 if($table_ok){
405 $tables = getTableList();
406 if(!array_key_exists($table, $tables)) $table_ok = false;
407 }
408
409 if(!$table_ok){
410 if($silent) return false;
411
412 echo $this->header();
413 echo errorMsg(str_replace('<TABLENAME>', html_attr($table), $this->lang['table name title']) . ': ' . $this->lang['does not exist'] . $this->error_back_link . $this->debug(__LINE__));
414 echo $this->footer();
415 return false;
416 }
417
418 return $table;
419 }
420
421 protected function table_fields($table){
422 $field_details = $fields = array();
423
424 $res = sql("show fields from `{$table}`", $eo);
425 while($row = db_fetch_assoc($res)){
426 $fields[] = $row['Field'];
427 $field_details[] = $row;
428 }
429
430 return $fields;
431 }
432
433 /**
434 * show js-driven preview of 1st 10 lines, with live csv options and column mapping options
435 */
436 public function show_preview(){
437
438 /* retrieve and validate table to import to */
439 $table = $this->get_table();
440 if(!$table) return;
441
442 /* retrieve fields of table */
443 $fields = $this->table_fields($table);
444
445 /* retrieve and open requested csv file */
446 $csv = $this->get_csv(array('htmlpage' => true));
447 if(!$csv) return;
448 $csv_fp = fopen("{$this->curr_dir}/csv/{$csv}", 'r');
449 if(!$csv_fp) return;
450
451 /* get the first 50 lines of the csv */
452 $lines = array();
453 $line_num = 0;
454 while(($line = fgets($csv_fp)) && $line_num < 50){
455 if(!$line_num) $line = trim($this->no_bom($line), '"');
456 $lines[] = trim($line);
457 $line_num++;
458 }
459
460 $lines_json = @json_encode($lines);
461 if($lines_json === false && function_exists('json_last_error')){
462 if(json_last_error() == JSON_ERROR_UTF8){
463 $lines = $this->utf8ize($lines);
464 $lines_json = @json_encode($lines);
465 }
466 }
467
468 echo $this->header();
469
470 if($lines_json === false){
471 $lines_json = '[]';
472 echo "\n<!-- \n\t" .
473 $this->debug(implode("\n\t", $lines), false) .
474 "\n -->\n";
475 if(function_exists('json_last_error')){
476 echo "\n<!-- \n\t" . $this->debug('json error: ' . json_last_error()) . "\n -->\n";
477 }
478 }
479
480 ?>
481
482 <script src="../resources/csv/jquery.csv.min.js"></script>
483
484 <div class="page-header"><h1><?php echo $this->lang['preview and confirm CSV data']; ?></h1></div>
485
486 <h3 class="text-right">
487 <?php echo $this->lang['CSV file']; ?>:
488 <span class="text-info"><?php echo html_attr($csv); ?></span>
489 <a href="<?php echo $this->curr_page; ?>" class="btn btn-warning" title="<?php echo html_attr($this->lang['cancel']); ?>"><i class="glyphicon glyphicon-remove"></i></a>
490 </h3>
491
492 <div class="row">
493 <div class="col-lg-offset-5 col-lg-7">
494 <div class="panel panel-success">
495 <div class="panel-heading">
496 <h3 class="panel-title text-center"><i class="glyphicon glyphicon-cog hspacer-md"></i> <?php echo $this->lang['change CSV settings']; ?></h3>
497 </div>
498 <div class="panel-body hidden">
499 <div id="csv-errors" class="alert alert-danger hidden">
500 <i class="glyphicon glyphicon-exclamation-sign"></i>
501 <?php echo $this->lang['error reading csv data']; ?>
502 </div>
503
504 <form class="form-horizontal" id="csv-settings">
505 <?php echo csrf_token(); ?>
506 <div class="form-group">
507 <div class="col-sm-8 col-md-6 col-lg-7 col-sm-offset-4 col-md-offset-3 col-lg-offset-4">
508 <label for="has_titles" class="control-label">
509 <input type="checkbox" id="has_titles" name="has_titles" value="1" checked>
510 <?php echo $this->lang['first line field names']; ?>
511 </label>
512 </div>
513 </div>
514 <div class="form-group">
515 <label for="ignore_lines" class="control-label col-sm-4 col-md-3 col-lg-4"><?php echo $this->lang['ignore lines number']; ?></label>
516 <div class="col-sm-8 col-md-6 col-lg-8">
517 <div class="input-group">
518 <input type="text" class="form-control" id="ignore_lines" name="ignore_lines" value="0">
519 <span class="input-group-btn">
520 <button class="btn btn-default" type="button" id="increment-ignored-lines"><i class="glyphicon glyphicon-plus"></i></button>
521 <button class="btn btn-default" type="button" id="decrement-ignored-lines"><i class="glyphicon glyphicon-minus"></i></button>
522 </span>
523 </div>
524 <span class="help-block"><?php echo $this->lang['skip lines number']; ?></span>
525 </div>
526 </div>
527 <div class="form-group">
528 <label for="field_separator" class="control-label col-sm-4 col-md-3 col-lg-4"><?php echo $this->lang['field separator']; ?></label>
529 <div class="col-sm-8 col-md-6 col-lg-8">
530 <input type="text" class="form-control" id="field_separator" name="field_separator" value=",">
531 <span class="help-block"><?php echo $this->lang['default comma']; ?></span>
532 </div>
533 </div>
534 <div class="form-group">
535 <label for="field_delimiter" class="control-label col-sm-4 col-md-3 col-lg-4"><?php echo $this->lang['field delimiter']; ?></label>
536 <div class="col-sm-8 col-md-6 col-lg-8">
537 <input type="text" class="form-control" id="field_delimiter" name="field_delimiter" value=""">
538 <span class="help-block"><?php echo $this->lang['default double-quote']; ?></span>
539 </div>
540 </div>
541 <div class="form-group">
542 <div class="col-sm-offset-4 col-sm-8 col-md-offset-3 col-md-9 col-lg-offset-4 col-lg-8">
543 <label class="">
544 <input type="checkbox" name="update_pk" id="update_pk" value="1">
545 <?php echo $this->lang['update table records'] ; ?>
546 </label>
547 <span class="help-block"><?php echo $this->lang['ignore CSV table records']; ?></span>
548 </div>
549 </div>
550 <div class="form-group">
551 <div class="col-sm-offset-4 col-sm-8 col-md-offset-3 col-md-9 col-lg-offset-4 col-lg-8">
552 <label class="">
553 <input type="checkbox" name="backup_table" id="backup_table" value="1" checked>
554 <?php echo $this->lang['back up the table'] ; ?>
555 </label>
556 </div>
557 </div>
558
559 <div class="row">
560 <div class="col-sm-4 col-sm-offset-4">
561 <button class="btn btn-warning btn-lg btn-block" type="button" id="reset-settings"><i class="glyphicon glyphicon-repeat"></i> <?php echo $this->lang['reset']; ?></button>
562 </div>
563 <div class="col-sm-4">
564 <button class="btn btn-info btn-lg btn-block" type="button" id="apply-settings"><i class="glyphicon glyphicon-ok"></i> <?php echo $this->lang['ok']; ?></button>
565 </div>
566 </div>
567 </form>
568 </div>
569 </div>
570 </div>
571 </div>
572
573 <div style="height: 5em;"></div>
574
575 <div class="table-responsive">
576 <table class="table table-striped table-hover table-bordered" id="csv-preview-table">
577 <thead><tr id="csv-fields"></tr><tr id="db-fields" class="bg-warning"></tr></thead>
578 <tbody>
579 </tbody>
580 </table>
581 </div>
582 <div class="alert alert-danger hidden" id="no-csv-data-error">
583 <i class="glyphicon glyphicon-exclamation-sign"></i>
584 <?php echo $this->lang['error reading csv data']; ?>
585 </div>
586
587 <div class="row">
588 <div class="col-sm-offset-7 col-sm-5 col-md-offset-8 col-md-4 col-lg-offset-9 col-lg-3">
589 <b class="text-danger hidden" id="mappings-warning"><i class="glyphicon glyphicon-info-sign"></i> <?php echo $this->lang['no columns selected']; ?></b>
590 <button type="button" class="hidden btn btn-primary btn-lg btn-block" id="start-import"><i class="glyphicon glyphicon-ok"></i> <?php echo $this->lang['import CSV']; ?></button>
591 </div>
592 </div>
593
594 <script>
595 $j(function(){
596 var csv_data = <?php echo $lines_json; ?>;
597 var fields = <?php echo json_encode($fields); ?>;
598
599 var default_config = {
600 field_separator: ',',
601 field_delimiter: '"',
602 ignore_lines: 0, // non-empty lines to ignore (after title line, if enabled)
603 has_titles: true
604 };
605 var config = $j.extend({}, default_config);
606
607 /* function to apply stored csv config to preview csv table */
608 var update_preview = function(){
609 if(!csv_data.length){
610 $j('#no-csv-data-error').removeClass('hidden');
611 return;
612 }
613 var csv_row = [], data_rows = 0, title_displayed= false;
614 var options = {
615 separator: config.field_separator,
616 delimiter: config.field_delimiter
617 }
618
619 /* clear table */
620 $j('#no-csv-data-error').addClass('hidden');
621 $j('#csv-fields,#db-fields,#csv-preview-table tbody').empty();
622
623 /* clear and hide errors */
624 $j('#csv-errors').addClass('hidden');
625
626 for(var i = 0; i < csv_data.length; i++){
627 if((data_rows - config.ignore_lines) >= 10) break;
628
629 try{
630 csv_row = $j.csv.toArray(csv_data[i], options);
631 }catch(e){
632 try{
633 /* BOM handling seems to strip the first " ni some cases */
634 csv_row = $j.csv.toArray('\"' + csv_data[i], options);
635 }catch(e){
636 $j('#csv-errors').removeClass('hidden');
637 continue;
638 }
639 }
640
641 /* disregard empty lines */
642 if((csv_row.length == 1 && csv_row[0] == '') || !csv_row.length) continue;
643
644 if(config.has_titles && !title_displayed){
645 add_table_header(csv_row);
646 title_displayed = true;
647 }else if(!config.has_titles && !title_displayed){
648 var generic_titles = [];
649 for(var j = 0; j < csv_row.length; j++){
650 generic_titles.push('<?php echo html_attr($this->lang['field']); ?> ' + (j + 1));
651 }
652 add_table_header(generic_titles)
653 title_displayed = true;
654
655 data_rows++;
656 if(data_rows > config.ignore_lines) add_table_row(csv_row);
657 }else{
658 data_rows++;
659 if(data_rows > config.ignore_lines) add_table_row(csv_row);
660 }
661 }
662 }
663
664 var add_table_row = function(row){
665 if(!row.length) return;
666 var rand_id = 'tr-' + Math.floor(Math.random() * 100000);
667 var td = '';
668 var num_columns = $j('tr:first th').length;
669
670 $j('#csv-preview-table tbody').append('<tr id="' + rand_id + '"></tr>');
671 for(var i = 0; i < num_columns; i++){
672 td = '<td>';
673 if(row[i] == undefined) row[i] = '';
674 if(row[i].length > 34) row[i] = row[i].substr(0, 30) + ' ...';
675 td += row[i] + '</td>';
676 $j('#' + rand_id).append(td);
677 }
678 }
679
680 var add_table_header = function(row){
681 for(var i = 0; i < row.length; i++){
682 $j('#csv-fields').append('<th>' + row[i] + '</th>');
683 $j('#db-fields').append('<th id="belongs-' + i + '"></th>');
684 render_belongs('#belongs-' + i, row[i]);
685 }
686 }
687
688 var import_button = function(hide_it){
689 if(!hide_it){
690 $j('#start-import').addClass('hidden');
691 $j('#mappings-warning').removeClass('hidden');
692 return;
693 }
694
695 $j('#start-import').removeClass('hidden');
696 $j('#mappings-warning').addClass('hidden');
697 }
698
699 var render_belongs = function(id, title){
700 var selected = '';
701 var csv_field_num = id.replace(/#belongs-/, '');
702
703 var dropdown = '<select class="form-control" id="db-field-for-' + csv_field_num + '">';
704 dropdown += '<option value="ignore-field"><<?php echo html_attr($this->lang['skip column']); ?>></option>';
705 for(var i = 0; i < fields.length; i++){
706 selected = '';
707 if(strip_name(fields[i]) == strip_name(title)){
708 selected = ' selected';
709 }
710 dropdown += '<option value="' + fields[i] + '"' + selected + '>' + fields[i] + '</option>';
711 }
712 dropdown += '</select>';
713
714 $j(id).append(
715 '<label for="db-field-for-' + csv_field_num + '" class="control-label">' +
716 '<?php echo html_attr($this->lang['belongs to']); ?>' +
717 '</label>' +
718 dropdown
719 );
720 }
721
722 /* function to strip strings */
723 var strip_name = function(name){
724 return name.toLowerCase().replace(/[\W_]+/g,'');
725 }
726
727 /* sync csv settings from stored values to screen */
728 var sync_screen_settings = function(){
729 $j('#field_separator').val(config.field_separator);
730 $j('#field_delimiter').val(config.field_delimiter);
731 $j('#ignore_lines').val(config.ignore_lines);
732 $j('#has_titles').prop('checked', config.has_titles);
733 }
734
735 /* sync csv settings from screen to stored values */
736 var sync_stored_settings = function(){
737 config.field_separator = $j('#field_separator').val();
738 config.field_delimiter = $j('#field_delimiter').val();
739 config.ignore_lines = parseInt($j('#ignore_lines').val());
740 if(config.ignore_lines < 0) config.ignore_lines = 0;
741 config.has_titles = $j('#has_titles').prop('checked');
742 }
743
744 /* parse GET parameters of the url and return the one requested */
745 var get = function(get_var){
746 var result = "",
747 tmp = [];
748 var items = location.search.substr(1).split("&");
749 for (var index = 0; index < items.length; index++){
750 tmp = items[index].split("=");
751 if (tmp[0] === get_var) result = decodeURIComponent(tmp[1]);
752 }
753 return result;
754 }
755
756 update_preview();
757
758 /* monitor column mappings and toggle import button accordingly */
759 setInterval(function(){
760 /* at least one field is selected for mapping? */
761 var mappings_exist = false;
762 $j('#db-fields select').each(function(){
763 if($j(this).val() != 'ignore-field'){
764 mappings_exist = true;
765 return false; // break loop
766 }
767 });
768
769 /* apply visual clues for mappings, checking for duplicates */
770 var mappings = {};
771 var no_duplicate_mapping = true;
772 $j('#db-fields select').each(function(){
773 if($j(this).val() == 'ignore-field'){
774 $j(this).parent().removeClass('bg-success bg-danger has-error');
775 return true; // continue loop
776 }
777
778 if(mappings[$j(this).val()] != undefined){
779 no_duplicate_mapping = false;
780 $j(this).parent().addClass('has-error bg-danger');
781 return false; // break loop
782 }
783 mappings[$j(this).val()] = true;
784 $j(this).parent().removeClass('has-error bg-danger').addClass('bg-success');
785 });
786
787 import_button(mappings_exist && no_duplicate_mapping);
788 }, 1000);
789
790 /* toggle panel-body and panel-footer on clicking panel-title */
791 $j('.panel-heading').click(function(){
792 var panel = $j(this).parent();
793 panel.find('.panel-body').toggleClass('hidden');
794 panel.find('.panel-footer').toggleClass('hidden');
795 });
796
797 /* close settings panel on clicking outside it */
798 $j('html, #apply-settings').click(function(){
799 $j('.panel-success .panel-body, .panel-success .panel-footer').addClass('hidden');
800 });
801 $j('.panel-success').click(function(e){
802 e.stopPropagation(); // don't bubble the click event to prevent closing the panel
803 });
804
805 /* reset csv settings to defaults */
806 $j('#reset-settings').click(function(){
807 config = $j.extend({}, default_config);
808 sync_screen_settings();
809 update_preview();
810 });
811
812 /* apply changes in csv settings to preview */
813 $j('#csv-settings').change(function(){
814 sync_stored_settings();
815 update_preview();
816 });
817
818 /* increase/decrease value in ignore_lines box */
819 $j('#increment-ignored-lines,#decrement-ignored-lines').click(function(){
820 var ignore_lines = parseInt($j('#ignore_lines').val());
821 if($j(this).attr('id') == 'increment-ignored-lines'){
822 ignore_lines++;
823 }else{
824 ignore_lines--;
825 }
826 if(ignore_lines < 0) ignore_lines = 0;
827 $j('#ignore_lines').val(ignore_lines).change();
828 });
829
830 /* prepare csv settings for submission on clicking the import button */
831 $j('#start-import').click(function(){
832 if(!$j('#csv-errors').hasClass('hidden')) return;
833
834 /* fetch csv settings */
835 var params = {
836 action: 'show_import_progress',
837 csv: get('csv'),
838 table: get('table'),
839 backup_table: $j('#backup_table').prop('checked') ? 1 : 0,
840 update_pk: $j('#update_pk').prop('checked') ? 1 : 0,
841 has_titles: $j('#has_titles').prop('checked') ? 1 : 0,
842 ignore_lines: Math.max(parseInt($j('#ignore_lines').val()), 0),
843 field_separator: $j('#field_separator').val(),
844 field_delimiter: $j('#field_delimiter').val(),
845 csrf_token: $j('#csrf_token').val()
846 };
847
848 /* fetch csv field mappings */
849 var num_fields= $j('#db-fields th').length;
850 for(var i = 0; i < num_fields; i++){
851 params['mappings[' + i + ']'] = $j('#db-field-for-' + i).val();
852 }
853
854 /* prepare submission url */
855 var url = '<?php echo $this->curr_page; ?>?' + $j.param(params);
856
857 /* disable submit for 60 seconds (timeout in case submission fails) */
858 $j(this).prop('disabled', true);
859 setTimeout(function(){
860 $j(this).prop('disabled', false);
861 }, 60000);
862
863 window.location = url;
864 });
865 });
866 </script>
867
868 <style>
869 .panel-heading{
870 cursor: pointer;
871 }
872 .panel-success{
873 width: 90%;
874 position: absolute;
875 opacity: .95;
876 right: 1.2em;
877 z-index: 999;
878 }
879 </style>
880
881 <?php
882 echo $this->footer();
883 }
884
885 /* ------------------------------------------------------ */
886
887 /**
888 * start/continue importing a csv file into the db (ajax-friendly)
889 */
890 public function import(){
891 @header('Content-type: application/json');
892 $res = array(
893 'imported' => 0,
894 'failed' => 0,
895 'remaining' => 0,
896 'logs' => array()
897 );
898
899 $csv_status = $this->start();
900 if(isset($csv_status['error'])){
901 $res['logs'][] = $csv_status['error'];
902 echo json_encode($res);
903 return;
904 }
905
906 $start = $csv_status['start'];
907
908 $lines = $this->csv_lines();
909 if($start >= $lines){ // no more rows to import
910 $res['logs'][] = $this->lang['mission accomplished'];
911 echo json_encode($res);
912 $this->start(0);
913 return;
914 }
915
916 $settings = $this->get_csv_settings();
917 if($settings === false){
918 $res['logs'][] = $this->debug(__LINE__, false) . $this->lang['csv file upload error'];
919 echo json_encode($res);
920 return;
921 }
922 $data_lines = $lines - ($settings['has_titles'] ? 1 : 0);
923
924 $bkp_res = $this->backup_table($start, $settings);
925 if(isset($bkp_res['error'])){
926 $res['logs'][] = $this->debug(__LINE__, false) . $bkp_res['error'];
927 echo json_encode($res);
928 return;
929 }elseif(isset($bkp_res['status'])){
930 $res['logs'][] = $bkp_res['status'];
931 }
932
933 $csv_data = $this->get_csv_data($start, $settings);
934 if(!count($csv_data)){
935 $res['logs'][] = $this->lang['mission accomplished'];
936 echo json_encode($res);
937 $this->start(0);
938 return;
939 }
940
941 $res['logs'][] = str_replace(
942 array('<RECORDNUMBER>', '<RECORDS>'),
943 array(number_format($start + 1), number_format($data_lines)),
944 $this->lang['start at estimated record']
945 );
946
947 $res['imported'] = count($csv_data);
948 $new_start = $start + $res['imported'];
949 $res['remaining'] = $lines - $new_start;
950
951 $query_info = $eo = array();
952 $insert = $this->get_query($csv_data, $settings, $query_info);
953 if($insert === false){
954 $res['logs'][] = $this->debug(__LINE__, false) . $this->lang['csv file upload error'];
955 echo json_encode($res);
956 return;
957 }
958
959 if(!sql($insert, $eo)){
960 $res['logs'][] = $this->debug(__LINE__, false) . db_error();
961 echo json_encode($res);
962 return;
963 }
964
965 $res['logs'][count($res['logs']) - 1] .= " {$this->lang['ok']}";
966
967 if($new_start >= $lines){
968 $this->start(0); /* reset csv status after finishing */
969 }else{
970 $this->start($new_start); /* update csv status file to new start */
971 }
972
973 echo json_encode($res);
974 }
975
976 protected function request_or($var, $default){
977 return (isset($this->request[$var]) ? $this->request[$var] : $default);
978 }
979
980 /**
981 * @brief Retrieve and validate CSV settings from REQUEST
982 *
983 * @return false on error, or associative array (table, backup_table, update_pk, has_titles, ignore_lines, field_separator, field_delimiter, mappings[])
984 */
985 protected function get_csv_settings(){
986 static $settings = array();
987 if(!empty($settings)) return $settings; // cache to avoid reprocessing
988
989 $settings = array(
990 'backup_table' => (bool) $this->request_or('backup_table', true),
991 'update_pk' => (bool) $this->request_or('update_pk', false),
992 'has_titles' => (bool) $this->request_or('has_titles', false),
993 'ignore_lines' => max(0, (int) $this->request_or('ignore_lines', 0)),
994 'field_separator' => $this->request_or('field_separator', ','),
995 'field_delimiter' => $this->request_or('field_delimiter', '"'),
996 'mappings' => $this->request_or('mappings', array())
997 );
998
999 if(!$settings['field_delimiter']) $settings['field_delimiter'] = '"';
1000 if(!$settings['field_separator']) $settings['field_separator'] = ',';
1001
1002 $settings['table'] = $this->get_table(true);
1003 if($settings['table'] === false) return false;
1004
1005 if(!is_array($settings['mappings']) || !count($settings['mappings'])) return false;
1006
1007 /* validate and trim field names */
1008 $last_field_key = count($settings['mappings']) - 1;
1009 $mappings = array();
1010 for($i = 0; $i <= $last_field_key; $i++){
1011 $fn = $settings['mappings'][$i];
1012 $fn = trim($fn);
1013 if(!preg_match('/^[a-zA-Z_][a-zA-Z0-9_]*$/', $fn) && $fn != 'ignore-field') return false;
1014
1015 /* make sure field is not already mapped to another column */
1016 if(isset($mappgins[$fn])) return false;
1017
1018 $settings['mappings'][$i] = $fn;
1019 if($fn == 'ignore-field'){
1020 unset($settings['mappings'][$i]);
1021 }else{
1022 $mappgins[$fn] = true;
1023 }
1024 }
1025
1026 if(!count($settings['mappings'])) return false;
1027
1028 return $settings;
1029 }
1030
1031 /**
1032 * @brief Counts non-empty lines in the current csv file. Count is cached for performance.
1033 *
1034 * @return number of non-empty data lines in csv
1035 *
1036 * @details This function counts all non-empty, including the title column
1037 */
1038 protected function csv_lines(){
1039 $csv = $this->get_csv();
1040 if(!$csv) return 0;
1041
1042 /*
1043 store lines count server-side:
1044 for each csv file being imported, create a count file in the csv folder named {csv-file-name.csv.count}
1045 the file stores the # of non-empty lines.
1046 */
1047 $count_file = "{$this->curr_dir}/csv/{$csv}.count";
1048 $csv_file = "{$this->curr_dir}/csv/{$csv}";
1049
1050 /* if csv modified after counting its lines, force recount */
1051 if(is_file($count_file) && filemtime($count_file) < filemtime($csv_file)){
1052 @unlink($count_file);
1053 return $this->csv_lines();
1054 }
1055
1056 $lines = @file_get_contents($count_file);
1057 if($lines !== false) return intval($lines);
1058
1059 /* this is a new import process */
1060 $lines = 0;
1061 $fp = @fopen($csv_file, 'r');
1062 if($fp === false) return 0;
1063
1064 /* start counting non-empty lines of csv */
1065 while($line = fgets($fp)){
1066 if(strlen(trim($line))) $lines++;
1067 }
1068 fclose($fp);
1069
1070 @file_put_contents($count_file, $lines);
1071 return $lines;
1072 }
1073
1074 /**
1075 * @brief if this is the beginning of the import process, perform table backup if requested
1076 *
1077 * @param $start current line in csv
1078 * @param $settings assoc arry of csv settings
1079 * @return assoc array, 'status' key and value if successful, 'error' key and value on error
1080 */
1081 protected function backup_table($start, $settings){
1082 if($start > 0) return array(); // no need to backup as we've passed the first batch
1083 if(!$settings['backup_table']) return array(); // no backup requested
1084
1085 $table = $this->get_table(true);
1086 if($table === false) return array('error' => $this->lang['no table name provided'] . $this->debug(__LINE__, false));
1087
1088 $stable = makeSafe($table);
1089 if(!sqlValue("select count(1) from `{$stable}`")) // nothing to backup!
1090 return array('status' => str_replace('<TABLE>', $table, $this->lang['table backup not done']));
1091
1092 $btn = $stable . '_backup_' . @date('YmdHis');
1093 $eo = array();
1094 sql("drop table if exists `{$btn}`", $eo);
1095 if(!sql("create table if not exists `{$btn}` like `{$stable}`", $eo))
1096 return array('error' => str_replace('<TABLE>', $table, $this->lang['error backing up table'] . $this->debug(__LINE__, false)));
1097 if(!sql("insert `{$btn}` select * from `{$stable}`", $eo))
1098 return array('error' => str_replace('<TABLE>', $table, $this->lang['error backing up table'] . $this->debug(__LINE__, false)));
1099
1100 return array(
1101 'status' => str_replace(
1102 array('<TABLE>', '<TABLENAME>'),
1103 array($table, $btn),
1104 $this->lang['table backed up']
1105 )
1106 );
1107 }
1108
1109 protected function no_bom($str){
1110 return preg_replace('/[\x00-\x1F\x80-\xFF]/', '', $str);
1111 }
1112
1113 /**
1114 * @brief opens current csv file and reads several lines from it, starting at non-empty line $start
1115 *
1116 * @param $start non-empty line # to start reading from
1117 * @param $settings csv settings array as retrieved from CSV::get_csv_settings
1118 * @return numeric 2D array of csv data (row1array, row2array, ...)
1119 */
1120 protected function get_csv_data($start, $settings){
1121 if($settings === false) return array();
1122
1123 $csv = $this->get_csv();
1124 $csv_file = "{$this->curr_dir}/csv/{$csv}";
1125 $first_line = true;
1126
1127 $fp = @fopen($csv_file, 'r');
1128 if(false === $fp) return array();
1129
1130 /* skip $start non-empty lines */
1131 $skip = $start;
1132
1133 /* apply ignore_lines */
1134 if($start < $settings['ignore_lines']) $skip = $settings['ignore_lines'];
1135
1136 /* get key of last mapping field -- used later here to apply ignored fields */
1137 end($settings['mappings']);
1138 $last_field_key = key($settings['mappings']);
1139
1140 /* skip title line */
1141 $skip += ($settings['has_titles'] ? 1 : 0);
1142
1143 for($i = 0; $i < $skip; $i++){
1144 /* keep reading till a non-empty line or EOF */
1145 do{
1146 $line = @implode('', @fgetcsv($fp));
1147 if($first_line){
1148 $line = $this->no_bom($line); /* remove BOM from 1st line */
1149 $first_line = false;
1150 }
1151 }while(trim($line) === '' && $line !== false);
1152
1153 if(false === $line){ fclose($fp); return array(); } /* EOF before $start */
1154 }
1155
1156 /* keep reading data from csv file till data size limit or EOF is reached */
1157 $csv_data = array(); $raw_data = '';
1158 do{
1159 $data = fgetcsv($fp, pow(2, 15), $settings['field_separator'], $settings['field_delimiter']);
1160 if($data === false){ fclose($fp); return $csv_data; } /* EOF */
1161
1162 if($first_line){
1163 $data[0] = $this->no_bom($data[0]); /* remove BOM if 1st line */
1164 $data[0] = trim($data[0], $settings['field_delimiter']); /* fix fgetcsv behavior with BOM */
1165 $first_line = false;
1166 }
1167 if(count($data) == 1 && !$data[0]) continue; /* empty line */
1168
1169 /* handle ignored fields */
1170 $last_key = max($last_field_key, count($data) - 1);
1171 for($i = 0; $i <= $last_key; $i++){
1172 if(!isset($data[$i])) $data[$i] = '';
1173 if(!isset($settings['mappings'][$i])) unset($data[$i]);
1174 }
1175
1176 $raw_data .= implode('', $data);
1177 $csv_data[] = $data;
1178 }while(strlen($raw_data) < $this->max_data_length && count($csv_data) < $this->max_batch_size);
1179
1180 fclose($fp);
1181 return $csv_data;
1182 }
1183
1184 /**
1185 * @brief Prepare the insert/replace query
1186 *
1187 * @param [in] $csv_data 2D numeric array of data to insert/replace
1188 * @param [in] $settings import settings assoc. array
1189 * @param [in,out] $query_info assoc array for exchanging query info and options
1190 * @return query string on success, false on error
1191 */
1192 protected function get_query(&$csv_data, $settings, &$query_info){
1193 /* make sure table name is provided */
1194 $table = $this->get_table(true);
1195 if($table === false){
1196 $query_info['error'] = $this->lang['no table name provided'] . $this->debug(__LINE__, false);
1197 return false;
1198 }
1199 $stable = makeSafe($table);
1200
1201 /* make sure mappings are provided */
1202 if(!isset($settings['mappings']) || !is_array($settings['mappings']) || !count($settings['mappings'])){
1203 $query_info['error'] = $this->lang['error reading csv data'] . $this->debug(__LINE__, false);
1204 return false;
1205 }
1206
1207 /* replace or insert? */
1208 $query = "INSERT IGNORE INTO ";
1209 if(isset($settings['update_pk']) && $settings['update_pk'] === true){
1210 $query = "REPLACE ";
1211 }
1212 $query .= "`{$stable}` ";
1213
1214 /* use mappings to determine field names */
1215 $query .= '(`' . implode('`,`', $settings['mappings']) . '`) VALUES ';
1216
1217 /* build query data */
1218 $insert_data = array();
1219 foreach($csv_data as $rec){
1220 /* sanitize data for SQL */
1221 foreach($rec as $i => $item){
1222 $rec[$i] = "'" . makeSafe($item, false) . "'";
1223 if($item === '') $rec[$i] = 'NULL';
1224 }
1225
1226 $insert_data[] = '(' . implode(',', $rec) . ')';
1227 }
1228 $query .= implode(",\n", $insert_data);
1229
1230 return $query;
1231 }
1232
1233 /**
1234 * get/set the next start of current csv file
1235 *
1236 * @param $start optional, new start value to save into status file
1237 * @return array('error' => error message) or array('start' => start line)
1238 */
1239 protected function start($new_start = false){
1240 $csv = $this->get_csv();
1241 if(!$csv){
1242 /* invalid csv file specified */
1243 return array('error' => $this->debug(__LINE__, false) . $this->lang['csv file upload error']);
1244 }
1245
1246 /*
1247 store progress server-side:
1248 for each csv file being imported, create a status file in the csv folder named {csv-file-name.csv.status}
1249 the file stores the last imported line#.
1250 */
1251 $status_file = "{$this->curr_dir}/csv/{$csv}.status";
1252 if(!is_file($status_file)){
1253 /* this is a new import process */
1254 /* create a status file and store $new_start into it */
1255 @file_put_contents($status_file, $new_start);
1256 }
1257
1258 if($new_start !== false && intval($new_start) >= 0){
1259 @file_put_contents($status_file, intval($new_start));
1260 return array('start' => intval($new_start));
1261 }
1262
1263 $start = @file_get_contents($status_file);
1264 if(false === $start){
1265 /* can't read file */
1266 return array('error' => $this->debug(__LINE__, false) . $this->lang['csv file upload error']);
1267 }
1268
1269 return array('start' => intval($start));
1270 }
1271
1272 /**
1273 * show page to control and monitor csv import process
1274 * (launch import job via ajax and keep relaunching and showing progress till done)
1275 */
1276 public function show_import_progress(){
1277 echo $this->header();
1278 if(!csrf_token(true)){
1279 echo errorMsg("{$this->lang['csrf token expired or invalid']}<br>{$this->error_back_link}" . $this->debug(__LINE__));
1280 echo $this->footer();
1281 return;
1282 }
1283 ?>
1284 <div class="page-header"><h1><?php echo $this->lang['importing CSV data']; ?></h1></div>
1285 <div class="progress">
1286 <div id="import-progress" class="progress-bar progress-bar-striped active progress-bar-info" style="width: 0">
1287 <span>0%</span>
1288 </div>
1289 </div>
1290
1291 <pre id="import-log"></pre>
1292
1293 <div id="next-action" class="hidden row">
1294 <div class="col-lg-offset-8 col-lg-4 col-md-offset-6 col-md-6 col-sm-offset-2 col-sm-8">
1295 <a href="pageAssignOwners.php" class="btn btn-success btn-lg btn-block"><i class="glyphicon glyphicon-user"></i> <?php echo "{$this->lang['next']}: {$this->lang['assign a records owner']}"; ?></a>
1296 </div>
1297 </div>
1298
1299 <div id="aborted" class="hidden alert-danger"><?php echo $this->lang['csv file upload error']; ?></div>
1300
1301 <script>
1302 $j(function(){
1303 var add_log = function(log_message){
1304 if(!log_message) return;
1305
1306 var import_log = $j("#import-log");
1307 import_log.append(log_message + '\n');
1308 import_log.scrollTop(import_log.prop("scrollHeight"));
1309 }
1310
1311 /* function to update progress */
1312 var update_progress = function(percent, log_message, status_class){
1313 if(isNaN(percent)) percent = 0;
1314
1315 /* limit to integer values between 0 and 100 */
1316 var p = Math.max(0, Math.min(parseInt(percent), 100));
1317 $j('#import-progress').css({ width: p + '%' });
1318 $j('#import-progress span').html(p + '%');
1319
1320 if(status_class != undefined){
1321 $j('#import-progress')
1322 .removeClass('progress-bar-success progress-bar-danger progress-bar-warning progress-bar-info progress-bar-primary')
1323 .addClass('progress-bar-' + status_class);
1324 }
1325
1326 if(status_class == 'danger' || p == 100){
1327 $j('#import-progress').removeClass('active');
1328 }
1329
1330 if(log_message != undefined) add_log(log_message);
1331 }
1332
1333 /**
1334 * function to trigger importing of a batch
1335 *
1336 * @param progress { total, failed, imported, retries }
1337 * @param callbacks { completed, aborted }
1338 *
1339 * @return Return_Description
1340 */
1341 var import_batch = function(progress, callbacks){
1342 // if first param is functions
1343 if(progress !== undefined){
1344 if($j.isFunction(progress.completed) || $j.isFunction(progress.aborted)){
1345 // we assume it's the callbacks
1346 callbacks = progress;
1347 progress = undefined;
1348 }
1349 }
1350
1351 progress = progress || { total: 0, failed: 0, imported: 0, retries: 0 };
1352
1353 var url = window.location.pathname + window.location.search.replace(/show_import_progress/, 'import');
1354
1355 $j.ajax({
1356 url: url
1357 }).done(function(data){
1358 /* data: { imported, failed, remaining, logs[] } */
1359 if(undefined == data.imported) data.imported = 0;
1360 if(undefined == data.failed) data.failed = 0;
1361 if(undefined == data.remaining) data.remaining = 0;
1362 if(undefined == data.logs) data.logs = [];
1363
1364 if(!progress.total) progress.total = data.imported + data.failed + data.remaining;
1365
1366 /* finished importing? */
1367 if(progress.total <= 0 || data.remaining <= 0){
1368 update_progress(100, '<b class="text-success"><?php echo html_attr($this->lang['finished status']); ?></b>', 'success');
1369 if(callbacks !== undefined && $j.isFunction(callbacks.completed)){
1370 callbacks.completed();
1371 }
1372 return;
1373 }
1374 progress.failed += data.failed;
1375 progress.imported += data.imported;
1376 progress.retries = 0;
1377
1378 update_progress((progress.failed + progress.imported) / progress.total * 100, data.logs.join('\n'));
1379 import_batch(progress, callbacks);
1380 }).fail(function(){
1381 /* if ajax failed, retry up to 10 times, with 10 seconds in-between then fail */
1382 if(progress.retries < 10){
1383 progress.retries++;
1384 update_progress((progress.failed + progress.imported) / progress.total * 100, '<?php echo html_attr(str_replace('<SECONDS>', '10', $this->lang['connection failed retrying'])); ?>', 'warning');
1385 setTimeout(function(){ import_batch(progress, callbacks); }, 3000);
1386 return;
1387 }else{
1388 /* fail and abort importing process */
1389 update_progress((progress.failed + progress.imported) / progress.total * 100, '<?php echo html_attr($this->lang['connection failed timeout']); ?>', 'danger');
1390 if(callbacks !== undefined && $j.isFunction(callbacks.aborted)){
1391 callbacks.aborted();
1392 }
1393 }
1394 });
1395 }
1396
1397 /* adjust import-log height based on window height */
1398 $j(window).resize(function(){
1399 $j('#import-log').height($j(window).height() * 0.4);
1400 }).resize();
1401
1402 add_log('<b class="text-warning"><?php echo html_attr($this->lang['please wait and do not close']); ?></b>');
1403 import_batch({
1404 completed: function(){
1405 $j('#next-action').removeClass('hidden');
1406 },
1407 aborted: function(){
1408 $j('#aborted').removeClass('hidden');
1409 }
1410 });
1411 })
1412 </script>
1413
1414 <style>
1415 #import-log{
1416 overflow: auto;
1417 }
1418 </style>
1419 <?php
1420 echo $this->footer();
1421 }
1422
1423 protected function header(){
1424 $Translation = $this->lang;
1425 ob_start();
1426 $GLOBALS['page_title'] = $Translation['importing CSV data'];
1427 include("{$this->curr_dir}/incHeader.php");
1428 $out = ob_get_contents();
1429 ob_end_clean();
1430
1431 return $out;
1432 }
1433
1434 protected function footer(){
1435 $Translation = $this->lang;
1436 ob_start();
1437 include("{$this->curr_dir}/incFooter.php");
1438 $out = ob_get_contents();
1439 ob_end_clean();
1440
1441 return $out;
1442 }
1443
1444 /**
1445 * @brief UTF8-encodes a string/array
1446 * @see https://stackoverflow.com/a/26760943/1945185
1447 *
1448 * @param [in] $mixed string or array of strings to be UTF8-encoded
1449 * @return UTF8-encoded array/string
1450 */
1451 protected function utf8ize($mixed) {
1452 if(is_array($mixed)){
1453 foreach($mixed as $key => $value){
1454 $mixed[$key] = $this->utf8ize($value);
1455 }
1456 }elseif(is_string($mixed)){
1457 return utf8_encode($mixed);
1458 }
1459 return $mixed;
1460 }
1461 }
